package book.crawler.bookText;

import book.crawler.CrawlerApplication;
import book.crawler.entity.Book;
import book.crawler.mapper.BookMapper;
import book.crawler.utils.HttpUtils;
import book.crawler.utils.TimeUtils;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import java.util.Scanner;
@Slf4j
@RunWith(SpringJUnit4ClassRunner.class)
@SpringBootTest(classes = CrawlerApplication.class)
/**
 * @author jiale
 * @version 1.0
 * @description这是一个爬取书城首页的算法
 * @date 2022/07/26 10:34
 **/
public class BookCrawler {
    String datetime = TimeUtils.format(System.currentTimeMillis(), "yyyy-MM-dd HH:mm:ss");
    private Scanner scan = new Scanner(System.in);
    private String url;
    private String html;
    private String title;
    private String author;
    private int num = 1;
    @Autowired
    private BookMapper mapper;
    @Autowired
    private Book book;

    @Test
    public void bookSource() throws InterruptedException {
        log.info("确认爬取小说吗？Y/N");
        String code = scan.nextLine();
        if ("Y".equals(code)) {
            url = "https://www.ireader.com.cn/index.php?ca=booksort.index&pca=booksort.index&pid=10";
            log.info("------------开始爬取!!!------------");
            html = HttpUtils.getHtml(url);//爬取分类首页
            Document doc1 = Jsoup.parse(html);
            Elements element1 = doc1.select("a:contains(频)");
            for (Element el1 : element1) {
                String gender = el1.text();
                book.setGender(gender);
                log.info("男女生?" + gender);
                url = el1.attr("href");
                System.out.println("主要网址:" + url);
                html = HttpUtils.getHtml(url);
                Document doc = Jsoup.parse(html);
                Elements element2 = doc.select("[data-id] a");
                for (Element el2 : element2) {
                    url = el2.attr("href");//因为“全部"这个标签无href标签，可以作为检测网址的条件
                    String type = el2.text();
                    book.setType(type);//设计类型：奇幻。。。
                    log.info("type:" + type);
                    for (int k = 0; k < 5; k++) {
                        String urls = url + k;//根据网址的status的值确定类型
                        switch (k) {//收费分类
                            case 0:
                                log.info("全部");
                                break;
                            case 1:
                                log.info("免费");
                                break;
                            case 2:
                                log.info("特价");
                                break;
                            case 3:
                                log.info("完结");
                                break;
                            default:
                                log.info("VIP");
                        }
                        while (num <= 1) {
                            urls += "&page=" + num;
                            html = HttpUtils.getHtml(urls);
                            doc = Jsoup.parse(html);
                            Elements element3 = doc.select(".show h3 a");
                            if (element3 == null) break;//有的分类下没有书籍
                            for (Element e : element3) {
                                String hl = HttpUtils.getHtml(e.attr("href"));
                                doc = Jsoup.parse(hl);
                                author = doc.select(".bookR .author").text();
                                author = author.substring(author.lastIndexOf("：") + 1);
                                book.setDatetime(datetime)//爬取时间
                                        .setAuthor(author)//作者
                                        .setUrl(doc.select(".bookL a img").attr("src"))//图片链接
                                        .setIntroduction(doc.select(".bookinf03 p").text())//简介
                                        .setClassify(doc.select(".bookL s").text());//分类
                                doc = Jsoup.parse(HttpUtils.getHtml(doc.select(".bookinf02 a").attr("href")));
                                title = doc.select("title").text();//标题
                                book.setTitle(title.substring(0, title.lastIndexOf("（掌阅）")))
                                        .setContent(doc.select(".article>p").text());//正文
                                mapper.bookinsert(book);//插入数据库
                                System.out.println(book);
                            }
                            num += 1;
                        }
                        num = 1;
                    }
                }
            }
        }
        log.info("------------爬取结束!!!------------");
    }
}







